{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#导入工具包\n",
    "import numpy as np\n",
    "import scipy.sparse as ss\n",
    "import scipy.io as sio\n",
    "\n",
    "import pickle\n",
    "\n",
    "#from utils import FeatureEng\n",
    "from sklearn.preprocessing import normalize\n",
    "\n",
    "import scipy.spatial.distance as ssd\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'lines = 0\\nfin = open(\"events.csv\",\\'rb\\') ###events.csv里面数目太大\\nfin.readline()\\nfor line in fin:\\n    cols = line.strip().split(\",\")\\n    lines += 1\\nfin.close()\\nprint(\"number of records : %d\" % lines)'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#统计活动数目\n",
    "'''lines = 0\n",
    "fin = open(\"events.csv\",'rb') ###events.csv里面数目太大\n",
    "fin.readline()\n",
    "for line in fin:\n",
    "    cols = line.strip().split(\",\")\n",
    "    lines += 1\n",
    "fin.close()\n",
    "print(\"number of records : %d\" % lines)'''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 读取之前算好的测试集和训练集中的活动"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of events in train & test :13418\n"
     ]
    }
   ],
   "source": [
    "#取出数据\n",
    "eventIndex = pickle.load(open(\"PE_eventIndex.pkl\",'rb'))\n",
    "#print(eventIndex)\n",
    "n_events = len(eventIndex)\n",
    "print(\"Number of events in train & test :%d\" % n_events)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#eventIndex"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "数据完美，总算把python3字符串与字节之间的编码问题解决了。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "b'event_id,user_id,start_time,city,state,zip,country,lat,lng,c_1,c_2,c_3,c_4,c_5,c_6,c_7,c_8,c_9,c_10,c_11,c_12,c_13,c_14,c_15,c_16,c_17,c_18,c_19,c_20,c_21,c_22,c_23,c_24,c_25,c_26,c_27,c_28,c_29,c_30,c_31,c_32,c_33,c_34,c_35,c_36,c_37,c_38,c_39,c_40,c_41,c_42,c_43,c_44,c_45,c_46,c_47,c_48,c_49,c_50,c_51,c_52,c_53,c_54,c_55,c_56,c_57,c_58,c_59,c_60,c_61,c_62,c_63,c_64,c_65,c_66,c_67,c_68,c_69,c_70,c_71,c_72,c_73,c_74,c_75,c_76,c_77,c_78,c_79,c_80,c_81,c_82,c_83,c_84,c_85,c_86,c_87,c_88,c_89,c_90,c_91,c_92,c_93,c_94,c_95,c_96,c_97,c_98,c_99,c_100,c_other\\n'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#FE = FeatureEng()\n",
    "fin = open(\"events.csv\",\"rb\")\n",
    "fin.readline()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<13418x101 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 0 stored elements in Dictionary Of Keys format>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#eventPropMatrix = ss.dok_matrix((n_events,7))\n",
    "eventCountMatrix = ss.dok_matrix((n_events,101))\n",
    "eventCountMatrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13418"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Count_eventId = 0\n",
    "for line in fin.readlines():\n",
    "    cols = line.decode().strip().split(\",\")\n",
    "    eventId = str(cols[0])\n",
    "    #print(eventId in eventIndex)#\n",
    "    if eventId in eventIndex:\n",
    "        Count_eventId += 1\n",
    "    #if eventIndex.has_key(eventId): #\n",
    "        i = eventIndex[eventId]\n",
    "        \n",
    "        #event的特征编码，这里只是简单处理，其实开始时间，地点等信息很重要\n",
    "        #eventPropMatrix[i,0] = FE.getJoinedYearMonth(cols[2])\n",
    "        #eventPropMatrix[i,1] = FE.getFeatureHash(col[3])\n",
    "        #eventPropMatrix[i,2] = FE.getFeatureHash(col[4])\n",
    "        #eventPropMatrix[i,3] = FE.getFeatureHash(col[5])\n",
    "        #eventPropMatrix[i,4] = FE.getFeatureHash(col[6])\n",
    "        #eventPropMatrix[i,5] = FE.getFeatureHash(col[7])\n",
    "        #eventPropMatrix[i,6] = FE.getFeatureHash(col[8])\n",
    "        \n",
    "        #词频，最后一个字段，其他关键字出现的频率不同\n",
    "        for j in range(9,109):\n",
    "            eventCountMatrix[i,j-9] = cols[j]\n",
    "            #print(cols[j])\n",
    "fin.close()\n",
    "Count_eventId"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<13418x101 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 186444 stored elements in Dictionary Of Keys format>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eventCountMatrix\n",
    "#查看eventCountMatrix有没有被赋值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<13418x101 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 186444 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#用L2横归一化，Kmeans聚类基于L2距离_\n",
    "#eventProMatrix = normalize(eventPropMatirx,norm='l2',copy=False)\n",
    "#sio.mmwrite(\"FE_eventCountMatrix\",eventPropMatrix)\n",
    "\n",
    "#词频，我们可以考虑用这部分特征进行聚类，得到活动的genre\n",
    "eventCountMatrix = normalize(eventCountMatrix, norm='l2',copy=False)\n",
    "sio.mmwrite(\"EV_eventCountMatix\",eventCountMatrix)\n",
    "eventCountMatrix"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
